#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import csv
import datetime as date
import requests
import re

#csv文件对应的头文件：stockCodesh：代码，stockNamesh：股票名称，stockNewPricesh：现在最新价格，stockUpDownsh：涨跌幅，stockUpDownchangesh涨跌额
#stockVolumesh:成交量,stockTurnoversh:成交额，stockAmplitudesh：振幅,stockHightpricesh':最高,stockOpenpricesh:开盘价，stockCloseyestsh：昨日收盘
#stockTurnoverratesh：换手率,stockP/Esh:市盈率,stockNetmarketsh:市净率
sh_stocks_days=['stockCodesh','stockNamesh','stockNewPricesh','stockUpDownsh','stockUpDownchangesh','stockVolumesh','stockTurnoversh','stockAmplitudesh',
                 'stockHightpricesh','stockLowerpricesh','stockOpenpricesh','stockCloseyestsh','stockTurnoverratesh','stockP/Esh','stockNetmarketsh']

#csv日线文件对应的头文件：'stockName':名称 'stockDatetimek':日期,'stockOpenpricek'：开盘价格,'stockClosepricek'：收盘价格,'stockHightpricek'：最高价格,'stockLowerpricek'：最低价格,'stockVolumek'：成交量,'stockTurnoverk'：换手率,
all_stocksdays_k=[
    'stockDatetimek','stockOpenpricek','stockClosepricek','stockHightpricek','stockLowerpricek','stockVolumek','stockTurnoverk',
    'stockAmplitudek'
]

#csv5分钟文件对应的头文件：'stockName':名称 'stockDatetimek':日期,'stockOpenprice5min'：开盘价格,'stockCloseprice5min'：收盘价格,'stockHightprice5min'：最高价格,'stockLowerprice5min'：最低价格,'stockVolume5min'：成交量,'stockTurnover5min'：换手率,
all_stocksdays_5min=[
    'stockDatetime5min','stockOpenprice5min','stockCloseprice5min','stockHightprice5min','stockLowerprice5min','stockVolume5min','stockTurnover5min',
    'stockAmplitude5min'
]

#csv1分钟文件对应的头文件：'stockName':名称 'stockDatetimek':日期,'stockOpenprice5min'：开盘价格,'stockCloseprice5min'：'stockTurnover5min'：换手率,     'stockother':其他
all_stocksdays_1min=[
    'stockDatetime5min','stockOpenprice1min','stockTurnover1min','stockCloseprice1min','stockother']
    #浏览器头文件
stock_headers={
    'User-Agent':'Mozilla/5.0 (X11; U; Linux x86_64; fr; rv:1.9.1.5) Gecko/20091109 Ubuntu/9.10 (karmic) Firefox/3.5.5'
}
# 建立今天股票的csv文件名称/Users/jundayan/Desktop/AI量化交易/quantizationprocharm/stockdata
stock_today_datetime = date.date.today().strftime('%Y%m%d')
stock_yesterday_datetime = (date.date.today() +date.timedelta(hours=-24)).strftime('%Y%m%d')
stock_today_csv = r'stockdata/'+stock_today_datetime + r'.csv'
stock_today_shcsv = r'stockdata/'+stock_today_datetime + r'sh.csv'
stock_today_zhcsv = r'stockdata/'+stock_today_datetime + r'zh.csv'
stock_today_newcsv = r'stockdata/'+stock_today_datetime + r'new.csv'
stock_today_mincsv = r'stockdata/'+stock_today_datetime + r'min.csv'
stock_today_gemcsv = r'stockdata/'+stock_today_datetime + r'gem.csv'
stock_today_kcbcsv = r'stockdata/'+stock_today_datetime + r'kcb.csv'
stock_today_stcsv = r'stockdata/'+stock_today_datetime + r'st.csv'
stock_today_sgcsv = r'stockdata/'+stock_today_datetime + r'sg.csv'
stock_today_warningcsv = r'stockdata/'+stock_today_datetime + r'warning.csv'
stock_day_lists=[]
stockpage_bool = True



#创建每日股票板块
def stockCreatedayallfile():
    global stockpage_bool,stock_day_lists
    # 股票所有股票价格
    #http://95.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112404882376022821404_1562591351041&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80,m:1+t:2&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562591351053
    if os.path.exists(stock_today_csv):
        todayexists = input('今天沪深所有股票的数据csv文件已存在，是否重新下载，是的（1），否（0）：')
        if (todayexists):
            stockCreatedayfile(
                r"http://95.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112404882376022821404_1562591351041&pn=",
                r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80,m:1+t:2&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562591351053",
                stock_today_csv)
    else:
        stockCreatedayfile(
            r"http://95.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112404882376022821404_1562591351041&pn=",
            r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80,m:1+t:2&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562591351053",
            stock_today_csv)

    # # 上海证券股票所有股票价格
    #http://89.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393505&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:1+t:2&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393617
    stockpage_bool = True
    stock_day_lists.clear()
    if os.path.exists(stock_today_shcsv):
        todayexists = input('今天沪所有股票的数据csv文件已存在，是否重新下载，是的（1），否（0）：')
        if (todayexists):
            stockCreatedayfile(
                r"http://89.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393505&pn=",
                r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:1+t:2&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393617",
                stock_today_shcsv)
    else:
        print('今天沪所有股票的数据csv文件:')
        stockCreatedayfile(
            r"http://89.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393505&pn=",
            r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:1+t:2&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393617",
            stock_today_shcsv)

        # # 深圳证券股票所有股票价格
        #http://41.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393606
        stockpage_bool = True
        stock_day_lists.clear()
        if os.path.exists(stock_today_zhcsv):
            todayexists = input('今天深圳证券股票有股票的数据csv文件已存在，是否重新下载，是的（1），否（0）：')
            if (todayexists):
                stockCreatedayfile(
                    r"http://41.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
                    r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393606",
                    stock_today_zhcsv)
        else:
            print('今天深圳证券股票的数据csv文件:')
            stockCreatedayfile(
                r"http://41.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
                r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393606",
                stock_today_zhcsv)
        #
    #
    # # 新股股票所有股票价格
    #http://48.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=m:0+f:8,m:1+f:8&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1562590393582
    stockpage_bool = True
    stock_day_lists.clear()
    if os.path.exists(stock_today_newcsv):
        todayexists = input('今天新股所有股票的数据csv文件已存在，是否重新下载，是的（1），否（0）：')
        if (todayexists):
            stockCreatedayfile(
                r"http://48.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
                r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=m:0+f:8,m:1+f:8&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1562590393582",
                stock_today_newcsv)
    else:
        print('今天新股所有股票的数据csv文件:')
        stockCreatedayfile(
            r"http://48.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
            r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=m:0+f:8,m:1+f:8&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1562590393582",
            stock_today_newcsv)
    #
    # # 中小板股票所有股票价格
    #http://66.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:13&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393571
    stockpage_bool = True
    stock_day_lists.clear()
    if os.path.exists(stock_today_mincsv):
        todayexists = input('今天中小板所有股票的数据csv文件已存在，是否重新下载，是的（1），否（0）：')
        if (todayexists):
            stockCreatedayfile(
                r"http://66.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
                r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:13&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393571",
                stock_today_mincsv)
    else:
        print('今天中小板所有股票的数据csv文件:')
        stockCreatedayfile(
            r"http://66.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
            r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:13&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393571",
            stock_today_mincsv)
    #

    # # 创业板股票所有股票价格
    #http://74.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:80&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393553
    stockpage_bool = True
    stock_day_lists.clear()
    if os.path.exists(stock_today_gemcsv):
        todayexists = input('今天创业板所有股票的数据csv文件已存在，是否重新下载，是的（1），否（0）：')
        if (todayexists):
            stockCreatedayfile(
                r"http://74.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
                r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:80&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393553",
                stock_today_gemcsv)
    else:
        print('今天创业板所有股票的数据csv文件:')
        stockCreatedayfile(
            r"http://74.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
            r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:80&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393553",
            stock_today_gemcsv)
    #
    # # 科创板股票所有股票价格
    #http://67.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:1+t:23&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393534
    stockpage_bool = True
    stock_day_lists.clear()
    if os.path.exists(stock_today_kcbcsv):
        todayexists = input('今天科创板所有股票的数据csv文件已存在，是否重新下载，是的（1），否（0）：')
        if (todayexists):
            stockCreatedayfile(
                r"http://67.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
                r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:1+t:23&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393534",
                stock_today_kcbcsv)
    else:
        print('今天科创板所有股票的数据csv文件:')
        stockCreatedayfile(
            r"http://67.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
            r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:1+t:23&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562590393534",
            stock_today_kcbcsv)


    # 泸股通股票所有股票价格
    #http://50.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=b:BK0707&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1562590393508
    stockpage_bool = True
    stock_day_lists.clear()
    if os.path.exists(stock_today_stcsv):
        todayexists = input('今天泸股通股票所有股票的数据csv文件已存在，是否重新下载，是的（1），否（0）：')
        if (todayexists):
            stockCreatedayfile(
                r"http://50.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
                r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=b:BK0707&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1562590393508",
                stock_today_stcsv)
    else:
        print('今天泸股通股票所有股票的数据csv文件:')
        stockCreatedayfile(
            r"http://50.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409584411100671866_1562590393503&pn=",
            r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=b:BK0707&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1562590393508",
            stock_today_stcsv)

    # 深股通股票所有股票价格
    #http://31.push2.eastmoney.com/api/qt/clist/get?cb=jQuery11240651645271237633_1562590184774&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=b:BK0804&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1562590184775
    stockpage_bool = True
    stock_day_lists.clear()
    if os.path.exists( stock_today_sgcsv):
        todayexists = input('今天深股通股票所有股票的数据csv文件已存在，是否重新下载，是的（1），否（0）：')
        if (todayexists):
            stockCreatedayfile(
                r"http://31.push2.eastmoney.com/api/qt/clist/get?cb=jQuery11240651645271237633_1562590184774&pn=",
                r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=b:BK0804&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1562590184775",
                stock_today_sgcsv)
    else:
        print('今天深股通股票所有股票的数据csv文件:')
        stockCreatedayfile(
              r"http://31.push2.eastmoney.com/api/qt/clist/get?cb=jQuery11240651645271237633_1562590184774&pn=",
              r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f26&fs=b:BK0804&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f26,f22,f11,f62,f128,f136,f115,f152&_=1562590184775",
              stock_today_sgcsv)

    # 风险提示股票所有股票价格
    #http://24.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112407130450132591215_1562588239233&pn=1&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+f:4,m:1+f:4&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562588239447
    stockpage_bool = True
    stock_day_lists.clear()
    if os.path.exists(stock_today_warningcsv):
        todayexists = input('今天风险提示股票所有股票的数据csv文件已存在，是否重新下载，是的（1），否（0）：')
        if (todayexists):
            stockCreatedayfile(
                r"http://24.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112407130450132591215_1562588239233&pn=",
                r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+f:4,m:1+f:4&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562588239447",
                stock_today_warningcsv)
    else:
        print('今天风险提示股票所有股票的数据csv文件:')
        stockCreatedayfile(
             r"http://24.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112407130450132591215_1562588239233&pn=",
             r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+f:4,m:1+f:4&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562588239447",
             stock_today_warningcsv)
#http://37.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112408365798924104935_1562521384726&pn=188&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80,m:1+t:2&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562521384727
# 创建包含header的csv名称
def  stockCreatedayfile(url1,url2,filename):

    stockday_num=1
    while stockpage_bool:
        # stockpage_url=r"http://37.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112408365798924104935_1562521384726&pn="+str(stockday_num)+r"&pz=20&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:13,m:0+t:80,m:1+t:2&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1562521384727"
        stockpage_url=url1+str(stockday_num)+url2
        stockPlatepages(stockpage_url)
        stockday_num=stockday_num+1



        # 创建包含header的csv名称
    with open(filename, 'w')as stock_today_fp:
        stock_today_write = csv.writer(stock_today_fp)
        stock_today_write.writerow(sh_stocks_days)
        stock_today_write.writerows(stock_day_lists)

        print(stock_day_lists)


    stock_today_fp.close()
#获取东方财富相关板块的总页数
def stockPlatepages(urls):
    stock_contents=requests.get(urls,headers=stock_headers).content.decode("utf-8")
    if re.search(r"null",stock_contents):
        global stockpage_bool,stockday_num
        stockday_num=0
        stockpage_bool = False
    else:
        stock_contentsjsonstr = re.search(r'.+data":(.+)]}}', stock_contents).group(1)
        stock_contentsjsondatastr=re.search(r'.+diff":\[(.+)',stock_contentsjsonstr ).group(1)
        stock_contentsjsondatastr=stock_contentsjsondatastr.replace(r'},',r'}?')
        stock_contentsjsondatastr_lists=stock_contentsjsondatastr.split(r"?")
        print(stock_contentsjsondatastr_lists)

        for stock_contentsjsondatastr_list in stock_contentsjsondatastr_lists:
            stock_contentsjsondatastr_dic =eval(stock_contentsjsondatastr_list)
            stock_simplestock_list=[]
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f12']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f14']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f2']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f3']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f4']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f5']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f6']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f7']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f15']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f16']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f17']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f18']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f8']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f9']))
            stock_simplestock_list.append(str(stock_contentsjsondatastr_dic['f23']))
            stock_day_lists.append(stock_simplestock_list)
            print(stock_simplestock_list)

#创建每日股票日k线文件夹
def stockCreatekallfile():
    if os.path.exists(r"stockdata/历史日k线数据"):
        pass
    else:
        os.mkdir(r"stockdata/历史日k线数据")

    stcok_yesterday_file=r'stockdata/'+stock_yesterday_datetime+r'.csv'


    with open(stcok_yesterday_file,'r')as stock_k_fp:
        stock_k_read=csv.DictReader(stock_k_fp)
        stock_simple_kurl1=r"http://pdfm.eastmoney.com/EM_UBG_PDTI_Fast/api/js?token=4f1862fc3b5e77c150a2b985b12db0fd&rtntype=6&id="
        stock_simple_kurl2=r"&type=k&authorityType=fa&cb=jsonp1562595479763"
        for stock_k_x  in stock_k_read:
            try:
                stockCreatedaysimplefile(stock_k_x['stockCodesh'],stock_k_x['stockNamesh'],stock_simple_kurl1,stock_simple_kurl2)
            except:
                print(stock_k_x['stockNamesh']+"此股已退出市场")

#分析每日股票k线文件夹是否已经创建和分析提取
def stockCreatedaysimplefile(stockcode,stcokname,url1,url2):
    print(stockcode+stcokname)
    global stock_simple_num
    if re.match(r"0",stockcode):
        stock_simple_num=r"2"

    else:
        if  re.match(r"3", stockcode):
            stock_simple_num = r"2"

        else:
            if re.match(r"6", stockcode):
                stock_simple_num = r"1"
    stock_simple_kurl = url1+stockcode+stock_simple_num+url2
    stock_simple_k =r"stockdata/历史日k线数据/"+str(stockcode)+r"k.csv"
    print(stock_simple_kurl)
    stock_simple_kcsv(stock_simple_kurl,stock_simple_k)

#分析每日股票k线文件夹分析提取
def stock_simple_kcsv(stock_simple_kurl,stock_simple_k):

      global stcok_simple_lastktie
      stock_simple_kcontent = requests.get( stock_simple_kurl,headers=stock_headers)
      stock_simple_kcontentstrs=stock_simple_kcontent.content.decode(r"utf-8")
      stock_simple_kcontentstrs=stock_simple_kcontentstrs.split(r":")
      for stock_simple_kcontentstr in stock_simple_kcontentstrs:
          stock_simple_kcontentstr=stock_simple_kcontentstr.lstrip()

          if re.match(r'\[',stock_simple_kcontentstr):


              stock_simple_kcontentstr = stock_simple_kcontentstr.replace(r'[', r"")
              stock_simple_kcontentstr = stock_simple_kcontentstr.replace(r']', r"")
              stock_simple_kcontentstr = stock_simple_kcontentstr.replace(r'flow', r"")
              stock_simple_kcontentstr = stock_simple_kcontentstr.strip()
              stock_simple_kcontentstrlists=stock_simple_kcontentstr.split(r'",')
              stock_simple_k_list=[]
              for stock_simple_kcontentstrlist in stock_simple_kcontentstrlists[:-1]:
                  stock_simple_kcontentstrlist= stock_simple_kcontentstrlist.replace(r'"',r"")
                  stock_simple_kcontentstrlist=stock_simple_kcontentstrlist.strip()
                  stock_simple_klists =stock_simple_kcontentstrlist.split(r',')
                  stock_simple_klist=[]
                  for x in stock_simple_klists:
                      x=x.strip()
                      stock_simple_klist.append(x)
                  stock_simple_k_list.append(stock_simple_klist)

              # for stock_simple_kcontentstrlist in stock_simple_kcontentstrlists
              with open(stock_simple_k, "w") as stock_simple_kwriter:
                  stock_simple_writer = csv.writer(stock_simple_kwriter)
                  stock_simple_writer.writerow(all_stocksdays_k)
                  stock_simple_writer.writerows(stock_simple_k_list)
              stock_simple_kwriter.close()
              break

#创建每日股票5分钟文件夹
def stockCreate5minallfile():
    if os.path.exists(r"stockdata/历史5分钟数据"):
        pass
    else:
        os.mkdir(r"stockdata/历史5分钟数据")

    stcok_yesterday_file=stock_yesterday_datetime+r'.csv'


    with open(stcok_yesterday_file,'r')as stock_k_fp:
        stock_k_read=csv.DictReader(stock_k_fp)
        stock_simple_kurl5min1=r"http://pdfm.eastmoney.com/EM_UBG_PDTI_Fast/api/js?token=4f1862fc3b5e77c150a2b985b12db0fd&rtntype=6&id="
        stock_simple_kurl5min2=r"&type=m5k&authorityType=fa&cb=jsonp1562595479763"
        for stock_k_x  in stock_k_read:
            try:
                stockCreatedaysimple5minfile(stock_k_x['stockCodesh'],stock_k_x['stockNamesh'],stock_simple_kurl5min1,stock_simple_kurl5min2)
            except:
                print(stock_k_x['stockCodesh']+"出现错误")

#分析每日股票5分钟文件夹是否已经创建和分析提取
def stockCreatedaysimple5minfile(stockcode,stcokname,url1,url2):
    print(stockcode+stcokname)
    global stock_simple_num
    if re.match(r"0",stockcode):
        stock_simple_num=r"2"

    else:
        if  re.match(r"3", stockcode):
            stock_simple_num = r"2"

        else:
            if re.match(r"6", stockcode):
                stock_simple_num = r"1"
    stock_simple_kurl = url1+stockcode+stock_simple_num+url2
    stock_simple_k =r"stockdata/历史5分钟数据/"+str(stockcode)+r"k.csv"
    print(stock_simple_kurl)
    stock_simple_5mincsv(stock_simple_kurl,stock_simple_k)

#分析每日股票5分钟文件夹分析提取
def stock_simple_5mincsv(stock_simple_kurl,stock_simple_k):

      global stcok_simple_lastktie

      stock_simple_kcontent = requests.get( stock_simple_kurl,headers=stock_headers)

      stock_simple_kcontentstrs=stock_simple_kcontent.content.decode(r"utf-8")

      stock_simple_kcontentstrs=stock_simple_kcontentstrs.split(r'":')
      for stock_simple_kcontentstr in stock_simple_kcontentstrs:
          stock_simple_kcontentstr=stock_simple_kcontentstr.lstrip()

          if re.match(r'\[',stock_simple_kcontentstr):


              stock_simple_kcontentstr = stock_simple_kcontentstr.replace(r'[', r"")
              stock_simple_kcontentstr = stock_simple_kcontentstr.replace(r']', r"")
              stock_simple_kcontentstr = stock_simple_kcontentstr.replace(r'flow', r"")
              stock_simple_kcontentstr = stock_simple_kcontentstr.strip()
              stock_simple_kcontentstrlists=stock_simple_kcontentstr.split(r'",')

              stock_simple_k_list=[]
              for stock_simple_kcontentstrlist in stock_simple_kcontentstrlists[:-1]:
                  stock_simple_kcontentstrlist= stock_simple_kcontentstrlist.replace(r'"',r"")
                  stock_simple_kcontentstrlist=stock_simple_kcontentstrlist.strip()
                  stock_simple_klists =stock_simple_kcontentstrlist.split(r',')
                  stock_simple_klist=[]
                  for x in stock_simple_klists:
                      x=x.strip()
                      stock_simple_klist.append(x)
                  stock_simple_k_list.append(stock_simple_klist)
              break
              print(stock_simple_k_list)

      if os.path.exists(stock_simple_k):
          stock_simple_lists5min=[]

          with open(stock_simple_k,"r") as stock_simple_reader:
              stock_simple_reader5mins = csv.reader(stock_simple_reader)

              for stock_simple_reader5min in stock_simple_reader5mins:
                  stock_simple_lists5min.append(stock_simple_reader5min )
          stock_simple_reader.close()
          stock_simple_boolstr=stock_simple_lists5min[-1][0]

          stock_bool=False
          stcok_bool1=False
          for stcok_simple_lastktie5min in stock_simple_k_list:
              if stock_bool:
                  stcok_bool1 = True
              if stcok_simple_lastktie5min[0]==stock_simple_boolstr:
                  stock_bool=True


              if stcok_bool1:
                  stock_simple_lists5min.append(stcok_simple_lastktie5min)
              else:
                  pass
          with open(stock_simple_k, "w") as stock_simple_kwriter:
              stock_simple_writer = csv.writer(stock_simple_kwriter)

              stock_simple_writer.writerows(stock_simple_lists5min)
          stock_simple_kwriter.close()

      else:
          # for stock_simple_kcontentstrlist in stock_simple_kcontentstrlists
          with open(stock_simple_k, "w") as stock_simple_kwriter:
              stock_simple_writer = csv.writer(stock_simple_kwriter)
              stock_simple_writer.writerow(all_stocksdays_k)

              stock_simple_writer.writerows(stock_simple_k_list)
          stock_simple_kwriter.close()

#创建每日股票1分钟文件夹
def stockCreate1minallfile():
    if os.path.exists(r"stockdata/历史1分钟数据"):
        pass
    else:
        os.mkdir(r"stockdata/历史1分钟数据")

    stcok_yesterday_file=stock_yesterday_datetime+r'.csv'


    with open(stcok_yesterday_file,'r')as stock_k_fp:
        stock_k_read=csv.DictReader(stock_k_fp)
        stock_simple_kurl1min1=r"http://pdfm.eastmoney.com/EM_UBG_PDTI_Fast/api/js?token=4f1862fc3b5e77c150a2b985b12db0fd&rtntype=5&id="
        stock_simple_kurl1min2=r"&type=r&iscr=false&cb=jsonp1562594960628"
        for stock_k_x  in stock_k_read:
            try:
                stockCreatedaysimple1minfile(stock_k_x['stockCodesh'],stock_k_x['stockNamesh'],stock_simple_kurl1min1,stock_simple_kurl1min2)
            except:
                print(stock_k_x['stockCodesh']+"出现错误")

#分析每日股票1分钟文件夹是否已经创建和分析提取
def stockCreatedaysimple1minfile(stockcode,stcokname,url1,url2):
    print(stockcode+stcokname)
    global stock_simple_num
    if re.match(r"0",stockcode):
        stock_simple_num=r"2"

    else:
        if  re.match(r"3", stockcode):
            stock_simple_num = r"2"

        else:
            if re.match(r"6", stockcode):
                stock_simple_num = r"1"
    stock_simple_kurl = url1+stockcode+stock_simple_num+url2
    stock_simple_k =r"stockdata/历史1分钟数据/"+str(stockcode)+r"k.csv"
    print(stock_simple_kurl)
    stock_simple_1mincsv(stock_simple_kurl,stock_simple_k)

#分析每日股票1分钟文件夹分析提取
def stock_simple_1mincsv(stock_simple_kurl,stock_simple_k):


      global stcok_simple_lastktie

      stock_simple_kcontent = requests.get( stock_simple_kurl,headers=stock_headers)

      stock_simple_kcontentstrs=stock_simple_kcontent.content.decode(r"utf-8")

      stock_simple_kcontentstrs=stock_simple_kcontentstrs.split(r'":')
      for stock_simple_kcontentstr in stock_simple_kcontentstrs:
          stock_simple_kcontentstr=stock_simple_kcontentstr.lstrip()

          if re.match(r'\[',stock_simple_kcontentstr):


              stock_simple_kcontentstr = stock_simple_kcontentstr.replace(r'[', r"")
              stock_simple_kcontentstr = stock_simple_kcontentstr.replace(r']', r"")
              stock_simple_kcontentstr = stock_simple_kcontentstr.replace(r'flow', r"")
              stock_simple_kcontentstr = stock_simple_kcontentstr.strip()
              stock_simple_kcontentstrlists=stock_simple_kcontentstr.split(r'",')

              stock_simple_k_list=[]
              for stock_simple_kcontentstrlist in stock_simple_kcontentstrlists[:-1]:
                  stock_simple_kcontentstrlist= stock_simple_kcontentstrlist.replace(r'"',r"")
                  stock_simple_kcontentstrlist=stock_simple_kcontentstrlist.strip()
                  stock_simple_klists =stock_simple_kcontentstrlist.split(r',')
                  stock_simple_klist=[]
                  for x in stock_simple_klists:
                      x=x.strip()
                      stock_simple_klist.append(x)
                  stock_simple_k_list.append(stock_simple_klist)
              break
              print(stock_simple_k_list)

      if os.path.exists(stock_simple_k):
          stock_simple_lists5min=[]

          with open(stock_simple_k,"r") as stock_simple_reader:
              stock_simple_reader5mins = csv.reader(stock_simple_reader)

              for stock_simple_reader5min in stock_simple_reader5mins:
                  stock_simple_lists5min.append(stock_simple_reader5min )
          stock_simple_reader.close()
          stock_simple_boolstr=stock_simple_lists5min[-1][0]

          stock_bool=False
          stcok_bool1=False
          for stcok_simple_lastktie5min in stock_simple_k_list:
              if stock_bool:
                  stcok_bool1 = True
              if not stcok_simple_lastktie5min[0]==stock_simple_boolstr:
                  stock_bool=True


              if stcok_bool1:
                  stock_simple_lists5min.append(stcok_simple_lastktie5min)
              else:
                  pass
          with open(stock_simple_k, "w") as stock_simple_kwriter:
              stock_simple_writer = csv.writer(stock_simple_kwriter)

              stock_simple_writer.writerows(stock_simple_lists5min)
          stock_simple_kwriter.close()

      else:
          # for stock_simple_kcontentstrlist in stock_simple_kcontentstrlists
          with open(stock_simple_k, "w") as stock_simple_kwriter:
              stock_simple_writer = csv.writer(stock_simple_kwriter)
              stock_simple_writer.writerow(all_stocksdays_1min)

              stock_simple_writer.writerows(stock_simple_k_list)
          stock_simple_kwriter.close()
